{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment: \n",
    "\n",
    "Analyze Hebbian Learning with different choices of hyperparameters\n",
    "- Different prune perc values\n",
    "- Hebbian Grow or not\n",
    "- Kwinners or RELU\n",
    "\n",
    "#### Motivation.\n",
    "\n",
    "- Verify if pruning by coactivations is better than pruning by magnitude\n",
    "- Verify if adding weights by coactivations is better than random\n",
    "\n",
    "#### Conclusion\n",
    "\n",
    "- Negative correlation between accuracy and hebbian pruning percentage - 0.4% diff in acc between base model (0.976) and pruning 30% every epoch by hebbian learning (0.972). In contrast to what is seen in the magnitude based pruning\n",
    "- Random growth outperforms hebbian pruning by ~ 0.2%\n",
    "- ReLU better than KWinners (with 25% on perc), by ~ 0.2%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from dynamic_sparse.common.browser import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = ['neurips_debug_test2', ]\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>model</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>num_classes</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>use_kwinners</th>\n",
       "      <th>weight_decay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_hebbian_grow=True,hebbian_prune_perc=0,use_k...</td>\n",
       "      <td>0.982033</td>\n",
       "      <td>29</td>\n",
       "      <td>0.908617</td>\n",
       "      <td>0</td>\n",
       "      <td>0.978142</td>\n",
       "      <td>0.982033</td>\n",
       "      <td>0.9725</td>\n",
       "      <td>18</td>\n",
       "      <td>0.9545</td>\n",
       "      <td>...</td>\n",
       "      <td>DSNNMixedHeb</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_hebbian_grow=False,hebbian_prune_perc=0,use_...</td>\n",
       "      <td>0.983183</td>\n",
       "      <td>27</td>\n",
       "      <td>0.913283</td>\n",
       "      <td>0</td>\n",
       "      <td>0.979750</td>\n",
       "      <td>0.982017</td>\n",
       "      <td>0.9749</td>\n",
       "      <td>11</td>\n",
       "      <td>0.9602</td>\n",
       "      <td>...</td>\n",
       "      <td>DSNNMixedHeb</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_hebbian_grow=True,hebbian_prune_perc=0.1,use...</td>\n",
       "      <td>0.971917</td>\n",
       "      <td>29</td>\n",
       "      <td>0.910133</td>\n",
       "      <td>0</td>\n",
       "      <td>0.965208</td>\n",
       "      <td>0.971917</td>\n",
       "      <td>0.9699</td>\n",
       "      <td>23</td>\n",
       "      <td>0.9523</td>\n",
       "      <td>...</td>\n",
       "      <td>DSNNMixedHeb</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_hebbian_grow=False,hebbian_prune_perc=0.1,us...</td>\n",
       "      <td>0.978083</td>\n",
       "      <td>14</td>\n",
       "      <td>0.908817</td>\n",
       "      <td>0</td>\n",
       "      <td>0.976125</td>\n",
       "      <td>0.974350</td>\n",
       "      <td>0.9741</td>\n",
       "      <td>11</td>\n",
       "      <td>0.9527</td>\n",
       "      <td>...</td>\n",
       "      <td>DSNNMixedHeb</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_hebbian_grow=True,hebbian_prune_perc=0.2,use...</td>\n",
       "      <td>0.967267</td>\n",
       "      <td>28</td>\n",
       "      <td>0.906567</td>\n",
       "      <td>0</td>\n",
       "      <td>0.960475</td>\n",
       "      <td>0.967167</td>\n",
       "      <td>0.9707</td>\n",
       "      <td>29</td>\n",
       "      <td>0.9514</td>\n",
       "      <td>...</td>\n",
       "      <td>DSNNMixedHeb</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Experiment Name  train_acc_max  \\\n",
       "0  0_hebbian_grow=True,hebbian_prune_perc=0,use_k...       0.982033   \n",
       "1  1_hebbian_grow=False,hebbian_prune_perc=0,use_...       0.983183   \n",
       "2  2_hebbian_grow=True,hebbian_prune_perc=0.1,use...       0.971917   \n",
       "3  3_hebbian_grow=False,hebbian_prune_perc=0.1,us...       0.978083   \n",
       "4  4_hebbian_grow=True,hebbian_prune_perc=0.2,use...       0.967267   \n",
       "\n",
       "   train_acc_max_epoch  train_acc_min  train_acc_min_epoch  train_acc_median  \\\n",
       "0                   29       0.908617                    0          0.978142   \n",
       "1                   27       0.913283                    0          0.979750   \n",
       "2                   29       0.910133                    0          0.965208   \n",
       "3                   14       0.908817                    0          0.976125   \n",
       "4                   28       0.906567                    0          0.960475   \n",
       "\n",
       "   train_acc_last  val_acc_max  val_acc_max_epoch  val_acc_min  ...  \\\n",
       "0        0.982033       0.9725                 18       0.9545  ...   \n",
       "1        0.982017       0.9749                 11       0.9602  ...   \n",
       "2        0.971917       0.9699                 23       0.9523  ...   \n",
       "3        0.974350       0.9741                 11       0.9527  ...   \n",
       "4        0.967167       0.9707                 29       0.9514  ...   \n",
       "\n",
       "          model  momentum  network  num_classes on_perc  optim_alg  \\\n",
       "0  DSNNMixedHeb       0.9   MLPHeb           10     0.2        SGD   \n",
       "1  DSNNMixedHeb       0.9   MLPHeb           10     0.2        SGD   \n",
       "2  DSNNMixedHeb       0.9   MLPHeb           10     0.2        SGD   \n",
       "3  DSNNMixedHeb       0.9   MLPHeb           10     0.2        SGD   \n",
       "4  DSNNMixedHeb       0.9   MLPHeb           10     0.2        SGD   \n",
       "\n",
       "   pruning_early_stop  test_noise use_kwinners weight_decay  \n",
       "0                   0       False         True       0.0001  \n",
       "1                   0       False         True       0.0001  \n",
       "2                   0       False         True       0.0001  \n",
       "3                   0       False         True       0.0001  \n",
       "4                   0       False         True       0.0001  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',\n",
       "       'data_dir', 'dataset_name', 'debug_sparse', 'debug_weights', 'device',\n",
       "       'hebbian_grow', 'hebbian_prune_perc', 'hidden_sizes', 'input_size',\n",
       "       'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler', 'model',\n",
       "       'momentum', 'network', 'num_classes', 'on_perc', 'optim_alg',\n",
       "       'pruning_early_stop', 'test_noise', 'use_kwinners', 'weight_decay'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72, 41)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name         1_hebbian_grow=False,hebbian_prune_perc=0,use_...\n",
       "train_acc_max                                                    0.983183\n",
       "train_acc_max_epoch                                                    27\n",
       "train_acc_min                                                    0.913283\n",
       "train_acc_min_epoch                                                     0\n",
       "train_acc_median                                                  0.97975\n",
       "train_acc_last                                                   0.982017\n",
       "val_acc_max                                                        0.9749\n",
       "val_acc_max_epoch                                                      11\n",
       "val_acc_min                                                        0.9602\n",
       "val_acc_min_epoch                                                       0\n",
       "val_acc_median                                                     0.9713\n",
       "val_acc_last                                                       0.9736\n",
       "epochs                                                                 30\n",
       "experiment_file_name    /Users/lsouza/nta/results/neurips_debug_test2/...\n",
       "trial_time                                                        18.5101\n",
       "mean_epoch_time                                                  0.617004\n",
       "batch_norm                                                           True\n",
       "data_dir                                        /home/ubuntu/nta/datasets\n",
       "dataset_name                                                        MNIST\n",
       "debug_sparse                                                         True\n",
       "debug_weights                                                        True\n",
       "device                                                               cuda\n",
       "hebbian_grow                                                        False\n",
       "hebbian_prune_perc                                                      0\n",
       "hidden_sizes                                                          100\n",
       "input_size                                                            784\n",
       "learning_rate                                                         0.1\n",
       "lr_gamma                                                              0.1\n",
       "lr_milestones                                                          60\n",
       "lr_scheduler                                                  MultiStepLR\n",
       "model                                                        DSNNMixedHeb\n",
       "momentum                                                              0.9\n",
       "network                                                            MLPHeb\n",
       "num_classes                                                            10\n",
       "on_perc                                                               0.2\n",
       "optim_alg                                                             SGD\n",
       "pruning_early_stop                                                      0\n",
       "test_noise                                                          False\n",
       "use_kwinners                                                         True\n",
       "weight_decay                                                       0.0001\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "DSNNMixedHeb    72\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # dataset related\n",
    "    dataset_name=\"MNIST\",\n",
    "    data_dir=os.path.expanduser(\"~/nta/datasets\"),\n",
    "    input_size=784,\n",
    "    num_classes=10,\n",
    "    # network related\n",
    "    network=\"MLPHeb\",\n",
    "    hidden_sizes=[100, 100, 100],\n",
    "    batch_norm=True,\n",
    "    use_kwinners=tune.grid_search([True, False]),\n",
    "    # model related\n",
    "    model=\"DSNNMixedHeb\",\n",
    "    on_perc=0.2,\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0.9,\n",
    "    weight_decay=1e-4,    \n",
    "    learning_rate=0.1,\n",
    "    lr_scheduler=\"MultiStepLR\",\n",
    "    lr_milestones=[30,60,90],\n",
    "    lr_gamma=0.1,\n",
    "    # sparse related\n",
    "    hebbian_prune_perc=tune.grid_search([0, 0.1, 0.2, 0.3, 0.4, 0.5]),\n",
    "    pruning_early_stop=0,\n",
    "    hebbian_grow=tune.grid_search([True, False]),\n",
    "    # additional validation\n",
    "    test_noise=False,\n",
    "    # debugging\n",
    "    debug_weights=True,\n",
    "    debug_sparse=True,\n",
    "    stop={\"training_iteration\": 30},\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<30][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72, 41)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed or incomplete trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=30]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: epochs, dtype: int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<30\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Which level of hebbian pruning was better?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>17</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.002</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>14</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.002</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>13</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.002</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>24</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.002</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>23</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.002</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>23</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.002</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                      model\n",
       "                          round_mean         min    max   mean    std count\n",
       "hebbian_prune_perc                                                         \n",
       "0.0                               17       0.972  0.978  0.976  0.002    12\n",
       "0.1                               14       0.970  0.978  0.973  0.002    12\n",
       "0.2                               13       0.968  0.975  0.972  0.002    12\n",
       "0.3                               24       0.968  0.975  0.971  0.002    12\n",
       "0.4                               23       0.967  0.973  0.970  0.002    12\n",
       "0.5                               23       0.967  0.973  0.970  0.002    12"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_prune_perc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>19</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>8</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>9</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>26</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>23</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>21</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                      model\n",
       "                          round_mean         min    max   mean    std count\n",
       "hebbian_prune_perc                                                         \n",
       "0.0                               19       0.976  0.978  0.977  0.000     6\n",
       "0.1                                8       0.971  0.978  0.974  0.002     6\n",
       "0.2                                9       0.971  0.975  0.973  0.002     6\n",
       "0.3                               26       0.971  0.975  0.972  0.001     6\n",
       "0.4                               23       0.970  0.973  0.971  0.001     6\n",
       "0.5                               21       0.969  0.973  0.971  0.001     6"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relu_only = (df['use_kwinners'] == False)\n",
    "agg(['hebbian_prune_perc'], relu_only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>16</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>21</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.003</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.2</th>\n",
       "      <td>16</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.3</th>\n",
       "      <td>21</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.4</th>\n",
       "      <td>24</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>25</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                      model\n",
       "                          round_mean         min    max   mean    std count\n",
       "hebbian_prune_perc                                                         \n",
       "0.0                               16       0.972  0.976  0.975  0.001     6\n",
       "0.1                               21       0.970  0.976  0.973  0.003     6\n",
       "0.2                               16       0.968  0.972  0.970  0.001     6\n",
       "0.3                               21       0.968  0.970  0.969  0.001     6\n",
       "0.4                               24       0.967  0.970  0.969  0.001     6\n",
       "0.5                               25       0.967  0.970  0.968  0.001     6"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kwinners_only = (df['use_kwinners'] == True)\n",
    "agg(['hebbian_prune_perc'], kwinners_only)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* No evidence of hebbian learning improving performance. Actually the opposite behavior, there is a clear reduction in acc as hebbian prune percentage increase from 0 to 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Did hebbian grow help?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_grow</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>17</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.003</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>22</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.001</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             val_acc_max_epoch val_acc_max                      model\n",
       "                    round_mean         min    max   mean    std count\n",
       "hebbian_grow                                                         \n",
       "False                       17       0.967  0.978  0.972  0.003    30\n",
       "True                        22       0.968  0.973  0.970  0.001    30"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with_pruning = (df['hebbian_prune_perc'] > 0)\n",
    "agg(['hebbian_grow'], with_pruning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Random Growth outperforms growing connections by using coactivations (hebbian grow) by 0.2%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th>hebbian_grow</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.0</th>\n",
       "      <th>False</th>\n",
       "      <td>16</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>19</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.1</th>\n",
       "      <th>False</th>\n",
       "      <td>13</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>16</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.2</th>\n",
       "      <th>False</th>\n",
       "      <td>6</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>20</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.3</th>\n",
       "      <th>False</th>\n",
       "      <td>23</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>25</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.4</th>\n",
       "      <th>False</th>\n",
       "      <td>22</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>25</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.5</th>\n",
       "      <th>False</th>\n",
       "      <td>20</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>26</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                val_acc_max_epoch val_acc_max                \\\n",
       "                                       round_mean         min    max   mean   \n",
       "hebbian_prune_perc hebbian_grow                                               \n",
       "0.0                False                       16       0.974  0.978  0.976   \n",
       "                   True                        19       0.972  0.977  0.976   \n",
       "0.1                False                       13       0.974  0.978  0.975   \n",
       "                   True                        16       0.970  0.973  0.971   \n",
       "0.2                False                        6       0.970  0.975  0.973   \n",
       "                   True                        20       0.968  0.972  0.970   \n",
       "0.3                False                       23       0.968  0.975  0.971   \n",
       "                   True                        25       0.969  0.972  0.971   \n",
       "0.4                False                       22       0.967  0.972  0.970   \n",
       "                   True                        25       0.968  0.973  0.970   \n",
       "0.5                False                       20       0.967  0.973  0.969   \n",
       "                   True                        26       0.968  0.971  0.970   \n",
       "\n",
       "                                       model  \n",
       "                                   std count  \n",
       "hebbian_prune_perc hebbian_grow               \n",
       "0.0                False         0.001     6  \n",
       "                   True          0.002     6  \n",
       "0.1                False         0.001     6  \n",
       "                   True          0.001     6  \n",
       "0.2                False         0.002     6  \n",
       "                   True          0.001     6  \n",
       "0.3                False         0.002     6  \n",
       "                   True          0.001     6  \n",
       "0.4                False         0.002     6  \n",
       "                   True          0.002     6  \n",
       "0.5                False         0.002     6  \n",
       "                   True          0.001     6  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with_pruning = (df['hebbian_prune_perc'] > 0)\n",
    "agg(['hebbian_prune_perc', 'hebbian_grow'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Which is better, kwinners or ReLU?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>use_kwinners</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>18</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>20</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             val_acc_max_epoch val_acc_max                      model\n",
       "                    round_mean         min    max   mean    std count\n",
       "use_kwinners                                                         \n",
       "False                       18       0.969  0.978  0.973  0.003    36\n",
       "True                        20       0.967  0.976  0.971  0.003    36"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['use_kwinners'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebbian_prune_perc</th>\n",
       "      <th>use_kwinners</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.0</th>\n",
       "      <th>False</th>\n",
       "      <td>19</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>16</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.1</th>\n",
       "      <th>False</th>\n",
       "      <td>8</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>21</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.003</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.2</th>\n",
       "      <th>False</th>\n",
       "      <td>9</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.002</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>16</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.3</th>\n",
       "      <th>False</th>\n",
       "      <td>26</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>21</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.4</th>\n",
       "      <th>False</th>\n",
       "      <td>23</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>24</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.5</th>\n",
       "      <th>False</th>\n",
       "      <td>21</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>25</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.001</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                val_acc_max_epoch val_acc_max                \\\n",
       "                                       round_mean         min    max   mean   \n",
       "hebbian_prune_perc use_kwinners                                               \n",
       "0.0                False                       19       0.976  0.978  0.977   \n",
       "                   True                        16       0.972  0.976  0.975   \n",
       "0.1                False                        8       0.971  0.978  0.974   \n",
       "                   True                        21       0.970  0.976  0.973   \n",
       "0.2                False                        9       0.971  0.975  0.973   \n",
       "                   True                        16       0.968  0.972  0.970   \n",
       "0.3                False                       26       0.971  0.975  0.972   \n",
       "                   True                        21       0.968  0.970  0.969   \n",
       "0.4                False                       23       0.970  0.973  0.971   \n",
       "                   True                        24       0.967  0.970  0.969   \n",
       "0.5                False                       21       0.969  0.973  0.971   \n",
       "                   True                        25       0.967  0.970  0.968   \n",
       "\n",
       "                                       model  \n",
       "                                   std count  \n",
       "hebbian_prune_perc use_kwinners               \n",
       "0.0                False         0.000     6  \n",
       "                   True          0.001     6  \n",
       "0.1                False         0.002     6  \n",
       "                   True          0.003     6  \n",
       "0.2                False         0.002     6  \n",
       "                   True          0.001     6  \n",
       "0.3                False         0.001     6  \n",
       "                   True          0.001     6  \n",
       "0.4                False         0.001     6  \n",
       "                   True          0.001     6  \n",
       "0.5                False         0.001     6  \n",
       "                   True          0.001     6  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['hebbian_prune_perc', 'use_kwinners'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* ReLU outperforms KWinners (with 25% perc) on all scenarios, by about 0.2%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
